/**
 * 
 */
package com.parser;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.StringReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpVersion;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicResponseHandler;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.beans.StringBean;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.joy.analyzer.html.Utility;
import org.mira.lucene.analysis.IK_CAnalyzer;

import com.Constant.Contants;

/**
 * @author LvSaiHui {gurnfe@163.com}
 * @version 1.0 <br>
 *          Copyright (C), 2007-2008, ZJUT <br>
 *          This program is protected by copyright laws. <br>
 *          Program Name: PageAnalysis.java <br>
 *          Date: Jan 4, 2009 <br>
 *          Description:
 */
public class PageAnalysis {
	public static Log log = LogFactory.getLog(PageAnalysis.class);

	public static String extractPlainText(String url) {

		StringBean sb = new StringBean();

		sb.setLinks(false);

		sb.setURL(url);

		String plainText = sb.getStrings();

		// System.out.println(plainText);
		return plainText;

	}

	public static Map<String, String> extractAllLinks(String url)
			throws IOException {
		Map<String, String> urls = new HashMap<String, String>();
		try {
			Parser parser = new Parser();
			/*
			 * Hashtable<String, String> properties= new Hashtable<String,
			 * String>(); properties.put("User-Agent",
			 * "Mozilla/4.0(compatible;MSIE 5.0;Windows NT;DigExt)");
			 * Parser.getConnectionManager().setRequestProperties(properties);
			 */

			URL myURL = new URL(url);
			URLConnection URLConn = (HttpURLConnection) myURL.openConnection();
			URLConn.setRequestProperty("User-Agent",
					"Mozilla/4.0(compatible;MSIE 5.0;Windows NT;DigExt)");
			parser.setConnection(URLConn);
			parser.setEncoding("gb2312");

			NodeFilter frameFilter = new NodeFilter() {

				private static final long serialVersionUID = 1L;

				public boolean accept(Node node) {
					if (node.getText().startsWith("frame src=")) {
						return true;
					} else {
						return false;
					}
				}
			};

			OrFilter linkFilter = new OrFilter(new NodeClassFilter(
					LinkTag.class), frameFilter);
			NodeList list = parser.extractAllNodesThatMatch(linkFilter);

			for (int i = 0; i < list.size(); i++) {
				Node tag = list.elementAt(i);
				if (tag instanceof LinkTag) {
					LinkTag link = (LinkTag) tag;
					String linkUrl = link.getLink();
					String text = link.getLinkText();
					urls.put(text, linkUrl);
					 System.out.println(text + " : " + linkUrl);
				} else {
					// <frame src="test.html"/>
					String frame = tag.getText();
					int start = frame.indexOf("src=");
					frame = frame.substring(start);
					int end = frame.indexOf(" ");
					if (end == -1)
						end = frame.indexOf(">");
					frame = frame.substring(5, end - 1);
					 System.out.println(frame);
					urls.put(Contants.frameTitle, frame);
				}
			}
		} catch (ParserException e) {
			e.printStackTrace();
		}
		/*
		 * Iterator it = urls.entrySet().iterator(); while (it.hasNext()) {//
		 * 健迭代 Map.Entry entry = (Map.Entry) it.next(); Object key =
		 * entry.getKey(); Object val = entry.getValue(); System.out.println(key
		 * + "--->" + val); }
		 */
		return urls;

	}

	public static String extractSourceCodeByHttpComponent(String url)
			throws ClientProtocolException, IOException {

		HttpClient httpclient = new DefaultHttpClient();

		HttpGet httpget = new HttpGet(url);
		HttpParams params = new BasicHttpParams();
		HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
		HttpProtocolParams.setContentCharset(params, "utf-8");
		HttpProtocolParams.setUserAgent(params, "HttpComponents/1.1");
		HttpProtocolParams.setUseExpectContinue(params, true);

		httpget.setParams(params);

		// System.out.println("executing request " + httpget.getURI());

		// Create a response handler
		ResponseHandler responseHandler = new BasicResponseHandler();
		String responseBody = (String) httpclient.execute(httpget,
				responseHandler);
		System.out.println(responseBody);
		return responseBody;
	}

	public static String extractSourceCode(String url) throws IOException {
		// URL myURL = new
		// URL("http://www.google.com.tw/search?hl=zh-TW&q=java&btnG=%E6%90%9C%E5%B0%8B&meta=");
		URL myURL = new URL(url);
		String buffer = "";
		String line = null;
		URLConnection URLConn = (HttpURLConnection) myURL.openConnection();
		URLConn.setRequestProperty("User-Agent",
				"Mozilla/4.0(compatible;MSIE 5.0;Windows NT;DigExt)");
		String openCharset =null;
		String charset = URLConn.getContentEncoding();
		// System.out.println(charset);
		String contentType = URLConn.getContentType();
		//System.out.println(contentType);
		String contentTypeCharset=null;
		if(contentType.indexOf("charset=")>0){
			contentTypeCharset=contentType.substring(contentType.indexOf("charset=")+8);
		}
		
		if (charset != null) {
			openCharset = charset;
		}
		else{
			if(contentTypeCharset!=null){
				openCharset=contentTypeCharset;
			}
			else{
				openCharset="UTF-8";
			}
		}

		BufferedReader htmlin = new BufferedReader(new InputStreamReader(
				URLConn.getInputStream(),openCharset));
		while ((line = htmlin.readLine()) != null) {
			buffer = buffer + line + "\n";
		}
		System.out.println(buffer);
		htmlin.close();
		return buffer;
	}

	public static String extractFeatureItem(String url) {
		String plainText = extractPlainText(url);
		log.info(plainText);
		log.info("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%");
		IK_CAnalyzer ika = new IK_CAnalyzer();
		try {
			System.out.println("Length = " + plainText.length());
			Reader r = new StringReader(plainText);
			TokenStream ts = ika.tokenStream("TestField", r);
			// long begin = System.currentTimeMillis();
			for (Token t = ts.next(); t != null; t = ts.next()) {
				System.out.println(t.startOffset() + " - " + t.endOffset()
						+ " = " + t.termText());
			}

			// long end = System.currentTimeMillis();
			// System.out.println("\u8017\u65F6 : " + (end - begin) + "ms");
		} catch (IOException e) {
			e.printStackTrace();
		}

		return null;
	}

	public static void saveSourceCodeToLocalDisk(String URL)
			throws ClientProtocolException, IOException, ParserException {
		

		String s = "";
		String encoding = "utf-8";
		try {
			// 从url打开stream
			InputStream in = null;
			HttpURLConnection conn = (HttpURLConnection) new URL(URL)
					.openConnection();
			in = conn.getInputStream();
			// 尝试从http头中获取字符集
			String contentType = conn.getHeaderField("Content-Type")
					.toLowerCase();
			
			boolean charsetFound = false;
			if (contentType.contains("charset")) {
				encoding = contentType.split("charset=")[1];
				charsetFound = true;
			}
			// 如果没有的话,读取头1024个字符，检查html的header
			byte[] buf = new byte[1024];
			if (!charsetFound) {
				int len = in.read(buf);
				while (len <= 32) {
					len = in.read(buf);
				}
				String header = new String(buf, 0, len);
				Pattern p = Pattern.compile(".*<meta.*content=.*charset=.*",
						Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
				Matcher m = p.matcher(header);
				if (m.matches()) {
					encoding = header.toLowerCase().split("charset=")[1]
							.replaceAll("[^a-z|1-9|\\-]", " ").split("\\s+")[0];
				} else {
					// 如果没有的话，直接用gb2312解码
					encoding = "gb2312";
				}
			}
			// 开始读取内容正文。
			BufferedReader br = new BufferedReader(new InputStreamReader(in,
					encoding));
			String header = new String(buf, encoding);
			// add the header to our content
			StringBuffer sb = new StringBuffer(header);
			char[] charBuf = new char[2048];
			int len = br.read(charBuf);
			while (len != -1) {
				sb.append(charBuf, 0, len);
				len = br.read(charBuf);
			}
			br.close();
			s = sb.toString();
			if (!s.trim().startsWith("<")) {
				s = "<" + s.trim();
			}
		} catch (IOException ex) {
			Logger.getLogger(Utility.class.getName()).log(Level.SEVERE, null,
					ex);
			throw new IOException();
		}
	
		//fetch the content completed
		
		String webPageTitle = null;
		Parser parser = new Parser();
		parser.setInputHTML(s);
		TagNameFilter filter = new TagNameFilter("title");
		NodeList nodeList = parser.extractAllNodesThatMatch(filter);
		if (nodeList.size() == 1) {
			// 因为文件的名字有要求，所以需要进一步过滤
			String temp = Utility.filterIllegerCharInFileName(nodeList
					.elementAt(0).toPlainTextString());
			if (temp != null && temp.trim().length() > 0) {
				webPageTitle = temp;
			}
		} else {
			webPageTitle = Contants.htmlName;
		}
		System.out.println("The title name in Html :" + webPageTitle);

		String destDir = Contants.htmlFileSavedDir + webPageTitle + ".html";

		File writefile = new File(destDir);
		if (!writefile.exists()) {
			writefile.createNewFile();
		}
		/*
		 * FileWriter fw = new FileWriter(writefile); PrintWriter out=new
		 * PrintWriter(fw,"UTF-8"); out.print(sourceCode); out.flush();
		 * out.close();
		 */
		OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(
				destDir),encoding);
		out.write(s, 0, s.length());
		out.close();

	}

	public static void main(String[] args) throws IOException, ParserException {
		// 一下测试通过,在网址前必须加 http://,
		 PageAnalysis.extractAllLinks("http://www.baidu.com");
		// PageAnalysis.extractSourceCodeByHttpComponent("http://www.163.com");
		 //PageAnalysis.saveSourceCodeToLocalDisk("http://topic.csdn.net/t/20010717/13/197347.html");
		 // PageAnalysis.saveSourceCodeToLocalDisk("http://www.163.com");
		// PageAnalysis.extractFeatureItem("http://news.sina.com.cn/c/2008-12-11/155516828944.shtml");
		// PageAnalysis.extractSourceCode("http://www.ccn86.com/news/adviser/20070612/23100.shtml");
		// PageAnalysis.extractSourceCode("http://wendell07.blog.hexun.com/26167967_d.html");

	}
}
